import aiohttp
import asyncio
import logging
import json
from motor.motor_asyncio import AsyncIOMotorClient

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

INDEX_URL = 'https://spa5.scrape.center/api/book/?limit=18&offset={offset}'
DETAIL_URL = 'https://spa5.scrape.center/api/book/{id}'
PAGE_SIZE = 18
PAGE_NUMBER = 100
CONCURRENCY = 10  # 并发量

MONGO_CONNECTION_STRING = 'mongodb://47.120.58.16:27017/'
MONGO_DB_NAME = 'books'
MONGO_COLLECTION_NAME = 'books'

client = AsyncIOMotorClient(MONGO_CONNECTION_STRING)
db = client[MONGO_DB_NAME]
collection = db[MONGO_COLLECTION_NAME]

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'
}

semaphore = asyncio.Semaphore(CONCURRENCY)
session = None


# 通用爬取方法
async def scrape_api(url):
    async with semaphore:
        try:
            logging.info('scraping %s', url)
            async with session.get(url, headers=headers) as response:
                return await response.json()
        except Exception as e:
            logging.error(e)


# 爬取列表页
async def scrape_index(page):
    url = INDEX_URL.format(offset=(page - 1) * PAGE_SIZE)
    return await scrape_api(url)


async def save_data(data):
    logging.info('saving data %s', data)
    if data:
        return await collection.update_one({'id': data['id']}, {'$set': data}, upsert=True)


async def scrape_detail(id):
    url = DETAIL_URL.format(id=id)
    data = await scrape_api(url)
    await save_data(data)


async def main():
    global session
    async with aiohttp.ClientSession() as session:
        scrape_index_tasks = [scrape_index(page) for page in range(1, PAGE_NUMBER + 1)]
        results = await asyncio.gather(*scrape_index_tasks)
        logging.info('results %s', json.dumps(results, ensure_ascii=False, indent=2))

        ids = []
        for index_data in results:
            if not index_data:
                continue
            for item in index_data['results']:
                ids.append(item['id'])

        scrape_detail_tasks = [scrape_detail(id) for id in ids]
        await asyncio.gather(*scrape_detail_tasks)


if __name__ == '__main__':
    asyncio.run(main())
